{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import csv\n",
    "# from concurrent.futures import (\n",
    "#     ThreadPoolExecutor, wait\n",
    "# )\n",
    "# from threading import Lock\n",
    "\n",
    "test_data_path = '../data/testData0626.csv'\n",
    "\n",
    "train_gps_path = '../data/train0523.csv'\n",
    "order_route_path = '../data/order_route_train.csv'\n",
    "complete_train_path = '../data/complete_train.csv'\n",
    "\n",
    "port_path = '../data/port.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_test_route_set(path):\n",
    "    data = pd.read_csv(path) \n",
    "    test_route_set = data['TRANSPORT_TRACE'].unique()\n",
    "    return test_route_set\n",
    "test_route_set = get_test_route_set(test_data_path)\n",
    "\n",
    "def get_port_info():\n",
    "    port_data = {}\n",
    "    test_port_set = set()\n",
    "    for route in test_route_set:\n",
    "        ports = route.split('-')\n",
    "        test_port_set = set.union(test_port_set, set(ports))\n",
    "    port_data_origin = pd.read_csv(port_path)\n",
    "    for item in port_data_origin.itertuples():\n",
    "        if getattr(item, 'TRANS_NODE_NAME') in test_port_set:\n",
    "            port_data[getattr(item, 'TRANS_NODE_NAME')] = {'LONGITUDE': getattr(item, 'LONGITUDE'),'LATITUDE': getattr(item, 'LATITUDE') }\n",
    "    return port_data\n",
    "port_data = get_port_info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "74it [23:50, 19.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- write csv done -\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "order_route = {}\n",
    "\n",
    "train_data_origin_chunk = pd.read_csv(train_gps_path, chunksize = 2000000, usecols = [0,1,2,3,4,5,6,7,12], header=None\n",
    "                                          , names=['loadingOrder', 'carrierName','timestamp','longitude','latitude','vesselMMSI','speed','direction','TRANSPORT_TRACE'])\n",
    "for chunk in tqdm(train_data_origin_chunk):\n",
    "    for row in chunk.itertuples(index=False):\n",
    "        for (port, port_info) in port_data.items():\n",
    "            if abs(row[3] - port_info['LONGITUDE']) < 0.3 and abs(row[4] - port_info['LATITUDE']) < 0.3:\n",
    "                if (not row[0] in order_route):\n",
    "                    order_route[row[0]] = []\n",
    "                if (not port in order_route[row[0]]):\n",
    "                    order_route[row[0]].append(port)\n",
    "\n",
    "with open(order_route_path,'w',encoding='utf-8',newline='') as f:\n",
    "    csv_writer = csv.writer(f)\n",
    "    for (order, order_info) in order_route.items():\n",
    "        if(len(order_info) == 1):\n",
    "            continue\n",
    "        csv_writer.writerow([order,'-'.join(order_info)])\n",
    "\n",
    "print('- write csv done -')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "74it [29:59, 24.32s/it]\n"
     ]
    }
   ],
   "source": [
    "train_data_origin_chunk = pd.read_csv(train_gps_path, chunksize = 2000000, usecols = [0,2,3,4,6,7,12], header=None,\n",
    "                                      names=['loadingOrder','timestamp','longitude','latitude','speed','direction','TRANSPORT_TRACE'])\n",
    "\n",
    "with open(complete_train_path,'w',encoding='utf-8',newline='') as f:\n",
    "    csv_writer = csv.writer(f)\n",
    "    for chunk in tqdm(train_data_origin_chunk):\n",
    "        for row in chunk.itertuples(index=False):\n",
    "            if (row[0] in order_route):\n",
    "                if row[5] == -1:\n",
    "                    continue\n",
    "                if(len(order_route[row[0]]) < 0):\n",
    "                    continue\n",
    "                edit_line = row._asdict()\n",
    "                edit_line.pop('direction')\n",
    "                edit_line['TRANSPORT_TRACE'] = '-'.join(order_route[row[0]])\n",
    "                list_values = [i for i in edit_line.values()]\n",
    "                csv_writer.writerow(list_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.6 64-bit ('AI': conda)",
   "language": "python",
   "name": "python37664bitaiconda6859e03b37c34f0182c9bde8073269f7"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
